Vehicle Detection Project
The goals / steps of this project are the following:
The code for this step is contained in lines 30 through 91 of the file called helper_functions.py.
I started by reading in all the vehicle and non-vehicle images.
Then I randomly picked an image from each classes and experienced with them.
I explored HOG feature extraction on the same image with different skimage.hog() parameters (orientations, pixels_per_cell, and cells_per_block).
| Parameters name | First set | Second set | Third set |
|---|---|---|---|
| Orientation | 9 | 9 | 9 |
| Pixels per cell | 8 | 4 | 16 |
| Cells per block | 2 | 2 | 2 |
| Visualize | True | True | True |
| Feature vector | False | False | False |
Import libraries
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
import glob
import random
% matplotlib inline
Import functions
from helper_functions import data_look
Initialization
# Images are in PNG format and divided into vehichles and non-vehicles
vehicles_images = glob.glob('./training_dataset/vehicles/*/*.png')
non_vehicles_images = glob.glob('./training_dataset/non-vehicles/*/*.png')
vehicles_all = []
non_vehicles_all = []
Reading all images
for image in vehicles_images:
vehicles_all.append(image)
for image in non_vehicles_images:
non_vehicles_all.append(image)
# Shuffle image data
random.shuffle(vehicles_all)
random.shuffle(non_vehicles_all)
# Take the first 100 images for testing
vehicles_test = vehicles_all[:4000]
non_vehicles_test = non_vehicles_all[:4000]
# Store some characteristics of the dataset
data_info_test = data_look(vehicles_test, non_vehicles_test)
print('Your function returned a count of',
data_info_test["n_vehicles"], ' vehicles and',
data_info_test["n_non_vehicles"], ' non-vehicles')
print('of size: ',data_info_test["image_shape"], ' and data type:',
data_info_test["data_type"])
print(len(vehicles_all))
print(len(non_vehicles_all))
augmented_vehicle_images = []
augmented_non_vehicle_images = []
# Augmented data set
for vehicle_img_path in vehicles_all:
# Read in vehicles
vehicle_image = mpimg.imread(vehicle_img_path)
augmented_vehicle_images.append(vehicle_image)
augmented_vehicle_images.append(cv2.flip(vehicle_image,1))
# Augmented data set
for non_vehicle_img_path in non_vehicles_all:
# Read in vehicles
non_vehicle_image = mpimg.imread(non_vehicle_img_path)
augmented_non_vehicle_images.append(non_vehicle_image)
augmented_non_vehicle_images.append(cv2.flip(non_vehicle_image,1))
print(len(augmented_vehicle_images))
print(len(augmented_non_vehicle_images))
Plot one sample image from each classes
# Return a random index within range of the minimum length in both lists
idx_max = np.minimum(data_info_test["n_vehicles"], data_info_test["n_non_vehicles"])
idx_random = np.random.randint(0, idx_max)
print("Current random index is", idx_random)
# Read in vehicles / non-vehicles images
vehicle_image_test = mpimg.imread(vehicles_test[idx_random])
non_vehicle_image_test = mpimg.imread(non_vehicles_test[idx_random])
# Plot the examples
f, axs = plt.subplots(1, 2, figsize=(30, 15))
axs = axs.ravel()
axs[0].imshow(vehicle_image_test)
axs[0].set_title('Example Vehicle Image', fontsize = 20)
axs[1].imshow(non_vehicle_image_test)
axs[1].set_title('Example Non-vehicle Image', fontsize = 20)
f.tight_layout()
plt.subplots_adjust(left=0., right=0.4, top=0.9, bottom=0.)
Apply HOG with different parameters
Import functions
from helper_functions import get_hog_features
Define parameters
# Define HOG parameters
# First set
orient = 9
pix_per_cell = 8
cell_per_block = 2
vis = True
feature_vec = False
# Second set
orient_2 = 9
pix_per_cell_2 = 4
cell_per_block_2 = 2
vis_2 = True
feature_vec_2 = False
# Third set
orient_3 = 9
pix_per_cell_3 = 16
cell_per_block_3 = 2
vis_3 = True
feature_vec_3 = False
Covert to grayscale
# Convert to grayscale
gray_vehicle_test = cv2.cvtColor(vehicle_image_test, cv2.COLOR_RGB2GRAY)
gray_non_vehicle_test = cv2.cvtColor(non_vehicle_image_test, cv2.COLOR_RGB2GRAY)
Return HOG features and visualization
# Return HOG features and visualization
# First set
features_vehicle, hog_image_vehicle = get_hog_features(gray_vehicle_test, orient,
pix_per_cell, cell_per_block,
vis, feature_vec)
features_non_vehicle, hog_image_non_vehicle = get_hog_features(gray_non_vehicle_test, orient,
pix_per_cell, cell_per_block,
vis, feature_vec)
# Second set
features_vehicle_2, hog_image_vehicle_2 = get_hog_features(gray_vehicle_test, orient_2,
pix_per_cell_2, cell_per_block_2,
vis_2, feature_vec_2)
features_non_vehicle_2, hog_image_non_vehicle_2 = get_hog_features(gray_non_vehicle_test, orient_2,
pix_per_cell_2, cell_per_block_2,
vis_2, feature_vec_2)
# Third set
features_vehicle_3, hog_image_vehicle_3 = get_hog_features(gray_vehicle_test, orient_3,
pix_per_cell_3, cell_per_block_3,
vis_3, feature_vec_3)
features_non_vehicle_3, hog_image_non_vehicle_3 = get_hog_features(gray_non_vehicle_test, orient_3,
pix_per_cell_3, cell_per_block_3,
vis_3, feature_vec_3)
Plot images
# Plot the examples
f, axs = plt.subplots(2, 4, figsize=(30, 30))
axs = axs.ravel()
axs[0].imshow(gray_vehicle_test)
axs[0].set_title('Grayscale Vehicle Image', fontsize = 20)
axs[1].imshow(hog_image_vehicle)
axs[1].set_title('HOG Visualization Vehicle Image', fontsize = 20)
axs[2].imshow(hog_image_vehicle_2)
axs[2].set_title('HOG Visualization Vehicle Image 2', fontsize = 20)
axs[3].imshow(hog_image_vehicle_3)
axs[3].set_title('HOG Visualization Vehicle Image 3', fontsize = 20)
axs[4].imshow(gray_non_vehicle_test)
axs[4].set_title('Grayscale Non-Vehicle Image', fontsize = 20)
axs[5].imshow(hog_image_non_vehicle)
axs[5].set_title('HOG Visualization Non-vehicle Image', fontsize = 20)
axs[6].imshow(hog_image_non_vehicle_2)
axs[6].set_title('HOG Visualization Non-vehicle Image 2', fontsize = 20)
axs[7].imshow(hog_image_non_vehicle_3)
axs[7].set_title('HOG Visualization Non-vehicle Image 3', fontsize = 20)
f.tight_layout()
plt.subplots_adjust(left=0.1, right=0.85, top=0.85, bottom=0.3)
Apply spatial binning and color histogram in different color spaces
Then I tried spatial binning and color histogram in different color spaces
| Parameters name | First set | Second set | Third set | Fourth set |
|---|---|---|---|---|
| Color spaces | 'RGb' | 'RGB' | 'HLS' | 'YCrCb' |
| Spatial size | (32, 32) | (32, 32) | (32, 32) | (32, 32) |
| Histogram bins | 32 | 32 | 32 | 32 |
| Histogram range | (0, 256) | (0, 256) | (0, 256) | (0, 256) |
Import libraries
# from sklearn.preprocessing import StandardScaler
# from mpl_toolkits.mplot3d import Axes3D
Import functions
# from helper_functions import extract_features_bin_spatial_hist, plot3d
Define parameters
# # RGB
# color_space = 'RGB'
# spatial_size=(32, 32)
# hist_bins=32
# hist_range=(0, 256)
# # HSV
# color_space_2 = 'RGB'
# spatial_size_2 = (32, 32)
# hist_bins_2 = 32
# hist_range_2 = (0, 256)
# # HLS
# color_space_3 = 'HLS'
# spatial_size_3 = (32, 32)
# hist_bins_3 = 32
# hist_range_3 = (0, 256)
# # YCrCb
# color_space_4 = 'YCrCb'
# spatial_size_4 = (32, 32)
# hist_bins_4 = 32
# hist_range_4 = (0, 256)
Return spatial binning of color features
# # RGB
# features_bin_spatial_hist_vehicle = extract_features_bin_spatial_hist(vehicles_test, color_space, spatial_size,
# hist_bins, hist_range)
# features_bin_spatial_hist_non_vehicle = extract_features_bin_spatial_hist(non_vehicles_test, color_space, spatial_size,
# hist_bins, hist_range)
# # HSV
# features_bin_spatial_hist_vehicle_2 = extract_features_bin_spatial_hist(vehicles_test, color_space_2, spatial_size_2,
# hist_bins_2, hist_range_2)
# features_bin_spatial_hist_non_vehicle_2 = extract_features_bin_spatial_hist(non_vehicles_test, color_space_2, spatial_size_2,
# hist_bins_2, hist_range_2)
# # HLS
# features_bin_spatial_hist_vehicle_3 = extract_features_bin_spatial_hist(vehicles_test, color_space_3, spatial_size_3,
# hist_bins_3, hist_range_3)
# features_bin_spatial_hist_non_vehicle_3 = extract_features_bin_spatial_hist(non_vehicles_test, color_space_3, spatial_size_3,
# hist_bins_3, hist_range_3)
# # YCrCb
# features_bin_spatial_hist_vehicle_4 = extract_features_bin_spatial_hist(vehicles_test, color_space_4, spatial_size_4,
# hist_bins_4, hist_range_4)
# features_bin_spatial_hist_non_vehicle_4 = extract_features_bin_spatial_hist(non_vehicles_test, color_space_4, spatial_size_4,
# hist_bins_4, hist_range_4)
Normalize feature vectors
# # RGB
# if len(features_bin_spatial_hist_vehicle) > 0:
# # Create an array stack of feature vectors
# X = np.vstack((features_bin_spatial_hist_vehicle, features_bin_spatial_hist_non_vehicle)).astype(np.float64)
# # Fit a per-column scaler
# X_scaler = StandardScaler().fit(X)
# # Apply the scaler to X
# scaled_X = X_scaler.transform(X)
# else:
# print('Your function only returns empty feature vectors for RGB')
# # HSV
# if len(features_bin_spatial_hist_vehicle_2) > 0:
# # Create an array stack of feature vectors
# X_2 = np.vstack((features_bin_spatial_hist_vehicle_2, features_bin_spatial_hist_non_vehicle_2)).astype(np.float64)
# # Fit a per-column scaler
# X_scaler_2 = StandardScaler().fit(X_2)
# # Apply the scaler to X
# scaled_X_2 = X_scaler_2.transform(X_2)
# else:
# print('Your function only returns empty feature vectors for HSV')
# # HLS
# if len(features_bin_spatial_hist_vehicle_3) > 0:
# # Create an array stack of feature vectors
# X_3 = np.vstack((features_bin_spatial_hist_vehicle_3, features_bin_spatial_hist_non_vehicle_3)).astype(np.float64)
# # Fit a per-column scaler
# X_scaler_3 = StandardScaler().fit(X_3)
# # Apply the scaler to X
# scaled_X_3 = X_scaler_3.transform(X_3)
# else:
# print('Your function only returns empty feature vectors for HLS')
# # YCrCb
# if len(features_bin_spatial_hist_vehicle_4) > 0:
# # Create an array stack of feature vectors
# X_4 = np.vstack((features_bin_spatial_hist_vehicle_4, features_bin_spatial_hist_non_vehicle_4)).astype(np.float64)
# # Fit a per-column scaler
# X_scaler_4 = StandardScaler().fit(X_4)
# # Apply the scaler to X
# scaled_X_4 = X_scaler_4.transform(X_4)
# else:
# print('Your function only returns empty feature vectors for YCrCb')
Plot images
# # Plot image
# fig = plt.figure(figsize=(12,4))
# # RGB
# plt.subplot(241)
# plt.plot(X[idx_random])
# plt.title('Raw Features(RGB)')
# plt.subplot(245)
# plt.plot(scaled_X[idx_random])
# plt.title('Normalized Features(RGB)')
# # HSV
# plt.subplot(242)
# plt.plot(X_2[idx_random])
# plt.title('Raw Features(HSV)')
# plt.subplot(246)
# plt.plot(scaled_X_2[idx_random])
# plt.title('Normalized Features(HSV)')
# # HLS
# plt.subplot(243)
# plt.plot(X_3[idx_random])
# plt.title('Raw Features(HLS)')
# plt.subplot(247)
# plt.plot(scaled_X_3[idx_random])
# plt.title('Normalized Features(HLS)')
# # YCrCb
# plt.subplot(244)
# plt.plot(X_4[idx_random])
# plt.title('Raw Features(YCrCb)')
# plt.subplot(248)
# plt.plot(scaled_X_4[idx_random])
# plt.title('Normalized Features(YCrCb)')
# fig.tight_layout()
Plot image in 3D
# # Read a color image
# img = cv2.imread(vehicles_test[idx_random])
# #img = cv2.imread("000275.png")
# #img = mpimg.imread(vehicles[idx_random])
# # Select a small fraction of pixels to plot by subsampling it
# scale = max(img.shape[0], img.shape[1], 64) / 64 # at most 64 rows and columns
# img_small = cv2.resize(img, (np.int(img.shape[1] / scale), np.int(img.shape[0] / scale)), interpolation=cv2.INTER_NEAREST)
# # Convert subsampled image to desired color space(s)
# # RGB
# img_small_RGB = cv2.cvtColor(img_small, cv2.COLOR_BGR2RGB) # OpenCV uses BGR, matplotlib likes RGB
# img_small_rgb = img_small_RGB / 255. # scaled to [0, 1], only for plotting
# # HSV
# img_small_HSV = cv2.cvtColor(img_small, cv2.COLOR_BGR2HSV)
# # HLS
# img_small_HLS = cv2.cvtColor(img_small, cv2.COLOR_BGR2HLS)
# # YCrCb
# img_small_YCrCb = cv2.cvtColor(img_small, cv2.COLOR_BGR2YCrCb)
# # Plot and show
# plot3d(img_small_RGB, img_small_rgb)
# plt.show()
# plot3d(img_small_HSV, img_small_rgb, axis_labels=list("HSV"))
# plt.show()
# plot3d(img_small_HLS, img_small_rgb, axis_labels=list("HLS"))
# plt.show()
# plot3d(img_small_YCrCb, img_small_rgb, axis_labels=list("YCrCb"))
# plt.show()
I tried various combinations of parameters and came up th following decision with explanation.
The documentation for hog() function can be found here.
orientation = 11.pix per cell = 8 is good enough.cell_per_block = 2.False to it.True to it.YUV color space with ALL channelsFinal list
orient_final = 11
pix_per_cell_final = 8
cell_per_block_final = 2
vis_final = False
feature_vec_final = True
color_space = 'YUV'
multichannel_final = 'ALL'
I trained a linear SVM using the following features:
Here are the training steps:
Import libraries
import time
import pickle
from sklearn.preprocessing import StandardScaler
from sklearn import svm
import sklearn.model_selection as grid_search
from sklearn.cross_validation import train_test_split
# But, if you are using scikit-learn >= 0.18 then use this:
# from sklearn.model_selection import train_test_split
Import functions
from helper_functions import extract_hog_features
Define parameters
# Define extraction parameters
color_space_final = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
spatial_size_final = (16, 16)
hist_bins_final = 16
hist_range_final = (0, 256)
orient_final = 11
pix_per_cell_final = 16
cell_per_block_final = 2
vis_final = False # Hardcoded in the function extract_features_single_image()
feature_vec_final = True # Hardcoded in the function extract_features_single_image()
multichannel_final = "ALL" # Can be 0, 1, 2, or "ALL"
spatial_feat = True # Spatial features on or off
hist_feat = True # Histogram features on or off
hog_feat = True # HOG features on or off
Extract the specific features from the dataset
t1=time.time()
# Create a list to append feature vectors to
vehicle_features_all = []
non_vehicle_features_all = []
# Extract features from each images
# for vehicle_image in augmented_vehicle_images:
# Extract features from image
vehicle_features_all = extract_hog_features(augmented_vehicle_images, cspace=color_space_final, orient=orient_final,
pix_per_cell=pix_per_cell_final, cell_per_block=cell_per_block_final,
hog_channel=multichannel_final)
t2 = time.time()
print(round(t2-t1, 2), 'Seconds to extract features from vehicle images.')
# Extract features from each images
# for non_vehicle_image in augmented_non_vehicle_images:
# Extract features from image
non_vehicle_features_all = extract_hog_features(augmented_non_vehicle_images, cspace=color_space_final, orient=orient_final,
pix_per_cell=pix_per_cell_final, cell_per_block=cell_per_block_final,
hog_channel=multichannel_final)
t3 = time.time()
print(round(t3-t2, 2), 'Seconds to extract features from non-vehicle images.')
print(len(vehicle_features_all))
print(len(non_vehicle_features_all))
Normalie the extracted features vector
# # Normalize
# if len(vehicle_features_all) > 0:
# # Create an array stack of feature vectors
# X_final = np.vstack((vehicle_features_all, non_vehicle_features_all)).astype(np.float64)
# # Fit a per-column scaler
# X_scaler_final = StandardScaler().fit(X_final)
# # Apply the scaler to X
# scaled_X_final = X_scaler_final.transform(X_final)
# else:
# print('Your function only returns empty feature vectors for RGB')
# Create an array stack of feature vectors
X_scaler_final = np.vstack((vehicle_features_all, non_vehicle_features_all)).astype(np.float64)
# Fit a per-column scaler - this will be necessary if combining different types of features (HOG + color_hist/bin_spatial)
#X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
#scaled_X = X_scaler.transform(X)
Define labels vector
# Define the labels vector
y_final = np.hstack((np.ones(len(vehicle_features_all)),
np.zeros(len(non_vehicle_features_all))))
Split up and randomize on both extracted features vector and labels vector
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
X_scaler_final, y_final, test_size=0.2, random_state=rand_state)
print('Using spatial binning of:',spatial_size_final,
'and', hist_bins_final,'histogram bins')
print('Feature vector length:', len(X_train[0]))
Create SVM
# Apply GridSearchCV to SVM
# parameters = {'kernel':('linear', 'rbf'), 'C':[0.01, 1]}
parameters = {'kernel':['linear'], 'C':[0.01, 1]}
# parameters = {'kernel':['linear'], 'C':[0.01]}
svr = svm.SVC()
# # Use a linear SVC
# svc = svm.LinearSVC()
# Check the training time for the SVC
t=time.time()
# svc.fit(X_train, y_train)
# Train the classifier
clf = grid_search.GridSearchCV(svr, parameters)
clf.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Return best fitted classifier
# The fit function now tries all the parameter combinations,
# and returns a fitted classifier that's automatically tuned to the optimal parameter combination.
# You can now access the parameter values via clf.best_params_
print('Optimal parameter combination is ', clf.best_params_)
svc = clf
Fit the training dataset into SVM
# Check the training time for the SVC
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
Evaluate the accuracy of SVM
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
Predict the result with the test dataset
# Check the prediction time for a single sample
t=time.time()
n_predict = 10
print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
Run the following code once a new classifier is created
Save a dictonary for future use
# Create dictionary
dict_vehicle_detection = {"svc": svc,
"X_scaler": X_scaler_final,
"orient": orient_final,
"pix_per_cell": pix_per_cell_final,
"cell_per_block": cell_per_block_final,
"feature_vec": feature_vec_final,
"spatial_size": spatial_size_final,
"hog_channel": multichannel_final,
"hist_bins": hist_bins_final,
"hist_range": hist_range_final,
"spatial_feat": spatial_feat,
"hist_feat": hist_feat,
"hog_feat": hog_feat,
}
# Save dictionary
pickle.dump(dict_vehicle_detection, open('dict_vehicle_detection.p', 'wb'))
I decided to search window in multiple different scales and set the cells_per_step = 2 based on the experiment below. The code below is used to evaluate the performance of different combination of window scale and overlap value.
From the final result, I come up the following conclusions:
I explored sliding window search on the same image with different parameteres.
| Parameters name | First set | Second set | Third set | Fourth set | Fifth set |
|---|---|---|---|---|---|
| Window size | (64, 64) | (64, 64) | (64, 64) | (32, 32) | (96, 96) |
| Window overlap | (0.5, 0.5) | (0.2, 0.2) | (0.8, 0.8) | (0.5, 0.5) | (0.5, 0.5) |
Import functions
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import cv2
import numpy as np
import pickle
import glob
% matplotlib inline
from helper_functions import search_windows, find_cars, draw_boxes, apply_threshold, draw_labeled_bboxes
Read in test images
test_img = mpimg.imread('./test_images/test1.jpg')
draw_image = np.copy(test_img)
# Uncomment the following line if you extracted training
# data from .png images (scaled 0 to 1 by mpimg) and the
# image you are searching is a .jpg (scaled 0 to 255)
#image = image.astype(np.float32)/255
Define parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
ystart = 350
ystop = 660
scale = 1.5
Load trained classifier
# Load dictionary
dict_pickle = pickle.load(open('dict_vehicle_detection.p', 'rb'))
# Read data
svc = dict_pickle["svc"]
Search windows with single scale
# Return the windows founded from the given image with single scale
draw_image, windows = find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, None, None, show_all_windows = False)
print(len(windows), 'windows found in image')
# Draw boxes
test_img_windows = draw_boxes(test_img, windows)
Plot images
plt.figure(figsize=(10,10))
plt.imshow(test_img_windows)
Search windows with multiple scales
# Return the windows founded from the given image with multiple scales
list_windows = search_windows(test_img, ystart, ystop, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, None, None, False)
# Learned from stackflow (https://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python)
windows_flatten = [item for sublist in list_windows for item in sublist]
# Draw boxes
test_img_windows = draw_boxes(test_img, windows_flatten, color='random', thick=2)
Plot images
plt.figure(figsize=(10,10))
plt.imshow(test_img_windows)
This method (duplicated from lesson material) produces a heat map based on rectangle locations (additive with overlap).
Import libraries
from scipy.ndimage.measurements import label
Import functions
from helper_functions import add_heat
Read in test images
# Images are in JPEG format
test_images = glob.glob('./test_images/test1.jpg')
# Creat a list to store the file paths
test_images_all = []
for image in test_images:
test_images_all.append(image)
Process test images
# Initialize an empty list
list_original_img = []
list_heatmap_original = []
list_heatmap_filtered = []
list_labeled_car = []
list_draw_img = []
# Read image
image = mpimg.imread(test_images[0])
heat_original = np.zeros_like(image[:,:,0]).astype(np.float)
# Add heat to each box in box list
heat = add_heat(heat_original,windows_flatten)
# Apply threshold to help remove false positives
heat_filtered = apply_threshold(np.copy(heat), 1)
# Visualize the heatmap when displaying
heatmap = np.clip(heat_filtered, 0, 255)
# Find final boxes from heatmap using label function
labels = label(heatmap)
# Draw bounding boxes on a copy of the image
draw_img = draw_labeled_bboxes(np.copy(image), labels)[0]
# Append the original image to the list
list_original_img.append(image)
# Append the original output heatmap to the list
list_heatmap_original.append(heat_original)
# Append the filtered output heatmap to the list
list_heatmap_filtered.append(heat_filtered)
# Append the output labels to the list
list_labeled_car.append(labels)
# Append the output image to the list
list_draw_img.append(draw_img)
Plot images
# Plot the draw images
f, axs = plt.subplots(5, 1, figsize=(36, 24))
# axs = axs.ravel()
# Plot image
axs[0].imshow(list_original_img[0])
# Set title name
title_name = 'Original'
axs[0].set_title(title_name, fontsize = 20)
axs[0].axis('off')
# Plot image
axs[1].imshow(list_heatmap_original[0], cmap='hot')
# Set title name
title_name = 'Original heatmap'
axs[1].set_title(title_name, fontsize = 20)
axs[1].axis('off')
# Plot image
axs[2].imshow(list_heatmap_filtered[0], cmap='hot')
# Set title name
title_name = 'Filtered heatmap'
axs[2].set_title(title_name, fontsize = 20)
axs[2].axis('off')
# Plot image
axs[3].imshow(list_labeled_car[0][0], cmap='gray')
# Set title name
title_name = 'Founded cars'
axs[3].set_title(title_name, fontsize = 20)
axs[3].axis('off')
# Plot image
axs[4].imshow(list_draw_img[0])
# Set title name
title_name = 'Draw images'
axs[4].set_title(title_name, fontsize = 20)
axs[4].axis('off')
f.tight_layout()
plt.subplots_adjust(left=0.1, right=0.85, top=0.85, bottom=0.3)
Ultimately I searched on multiple scales using HLS 3-channel HOG features plus spatially binned color and histograms of color in the feature vector, which provided a nice result. Those features extraction, classifying and window drawing were implemented in the function find_cars(). To enhance the efficiency of classifying, there are few code in find_cars() to extract HOG features just once for the entire region of interest in each full image / video frame. Please refer to the below images for your reference.
Import libraries
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
import glob
import random
% matplotlib inline
Import functions
from helper_functions import process_test_image
Read in test images
# Images are in JPEG format
test_images = glob.glob('./test_images/*.jpg')
# Creat a list to store the file paths
test_images_all = []
for image in test_images:
test_images_all.append(image)
Plot founded cars on test images
fig, axs = plt.subplots(3, 2, figsize=(16,14))
fig.subplots_adjust(hspace = .004, wspace=.002)
axs = axs.ravel()
for i, im in enumerate(test_images):
axs[i].imshow(process_test_image(mpimg.imread(im)))
axs[i].axis('off')
Here's a link to my video result
Import Libraries
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
from helper_functions import process_image
Process Video
Test on the given video project_video.mp4
video_output = 'test_videos_output/project_video_output.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
# clip1 = VideoFileClip("project_video.mp4").subclip(4,10)
# clip1 = VideoFileClip("project_video.mp4").subclip(4,15)
# clip1 = VideoFileClip("project_video.mp4").subclip(23,30)
# clip1 = VideoFileClip("project_video.mp4").subclip(23,38)
# clip1 = VideoFileClip("project_video.mp4").subclip(4,20)
clip1 = VideoFileClip("project_video.mp4")
video_output_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time video_output_clip.write_videofile(video_output, audio=False)
Play Video Inline
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(video_output))
I recorded the multiple scales position of positive detection in each frame of the video. From the positive detections I created a heatmap and then thresholded that map to identify vehicle positions. I then used scipy.ndimage.measurements.label() to identify individual blobs in the heatmap. I then assumed each blob corresponded to a vehicle. I constructed bounding boxes to cover the area of each blob detected.
Below are six drawn test images, their corresponding threshold images and the final heatmap.
Import librarires
from collections import deque
Import functions
def process_image_demo(img):
# Load dictionary
dict_pickle = pickle.load(open('dict_vehicle_detection.p', 'rb'))
# Read data
svc = dict_pickle["svc"]
# Define parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
ystart = 350
ystop = 660
# Search windows
list_windows = search_windows(img, ystart, ystop, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, None, None, False)
# Learned from stackflow (https://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python)
windows_flatten = [item for sublist in list_windows for item in sublist]
# Create heatmap
heat_original = np.zeros_like(img[:,:,0]).astype(np.float)
# Add heat to each box in box list
heat = add_heat(heat_original,windows_flatten)
# Add new heat map to the list
heatmaps_demo.append(np.copy(heat))
# Define threshold value
threshold_filter_demo = 10
# Apply threshold to help remove false positives
combined = sum(heatmaps_demo)
heat_filtered = apply_threshold(combined, threshold_filter_demo)
# Visualize the heatmap when displaying
heatmap = np.clip(heat_filtered, 0, 255)
# Find final boxes from heatmap using label function
labels = label(heatmap)
# Draw bounding boxes on a copy of the image
draw_img = draw_labeled_bboxes(np.copy(img), labels)[0]
return heat_original, heat_filtered, labels[0], draw_img
Read images
# Images are in png format
test_series_images = glob.glob('./output_images/test_1/*.png')
# Creat a list to store the file paths
test_series_images_all = []
for image in test_series_images:
test_series_images_all.append(image)
Run functions
n_frames_demo = 4
heatmaps_demo = deque(maxlen = n_frames_demo)
# Initialize an empty list
list_test_series_images = []
list_heatmap_original = []
list_heatmap_filtered = []
list_labeled_car = []
list_draw_img = []
for idx in range(len(test_series_images_all)):
# Read image
image = mpimg.imread(test_series_images_all[idx])
# Run function
heat_original, heat_filtered, labeled_car, draw_img = process_image_demo(np.copy(image))
# Append the original image to the list
list_test_series_images.append(image)
# Append the original heatmap to the list
list_heatmap_original.append(heat_original)
# Append the filtered heatmap to the list
list_heatmap_filtered.append(heat_filtered)
# Append the labeled car to the list
list_labeled_car.append(labeled_car)
# Append the output image to the list
list_draw_img.append(draw_img)
Plot images
# Plot the draw images
col = len(test_series_images_all)
f, axs = plt.subplots(5, col, figsize=(36, 24))
axs = axs.ravel()
for idx in range(col):
# Plot image
axs[idx].imshow(list_test_series_images[idx])
# Set title name
title_name = 'Original'
axs[idx].set_title(title_name, fontsize = 20)
axs[idx].axis('off')
# Plot image
axs[idx+col].imshow(list_heatmap_original[idx], cmap='hot')
# Set title name
title_name = 'Original heatmap'
axs[idx+col].set_title(title_name, fontsize = 20)
axs[idx+col].axis('off')
# Plot image
axs[idx+col*2].imshow(list_heatmap_filtered[idx], cmap='hot')
# Set title name
title_name = 'Filtered heatmap'
axs[idx+col*2].set_title(title_name, fontsize = 20)
axs[idx+col*2].axis('off')
# Plot image
axs[idx+col*3].imshow(list_labeled_car[idx], cmap='gray')
# Set title name
title_name = 'Founded cars'
axs[idx+col*3].set_title(title_name, fontsize = 20)
axs[idx+col*3].axis('off')
# Plot image
axs[idx+col*4].imshow(list_draw_img[idx])
# Set title name
title_name = 'Draw images'
axs[idx+col*4].set_title(title_name, fontsize = 20)
axs[idx+col*4].axis('off')
f.tight_layout()
plt.subplots_adjust(left=0.1, right=0.85, top=0.85, bottom=0.3)
Here are my steps to approach to the final pipeline.
Created my pipeline architecture with a small amout of test data
Tried different combination of color spaces and HOG features.
Here are my problems/issues
Updated at 02/18/2018: